Data Preprocess2

In [2]:

    

from six.moves import cPickle as pickle
import matplotlib.pyplot as plt
import os
from random import sample, shuffle
import numpy as np
import deepdish as dd


    

In [5]:

    

files = os.listdir('new_pickle_mean')
dataset = dict()
for file_name in files:
    with open('new_pickle_mean/'+file_name, 'rb') as f:
        save = pickle.load(f)
        dataset.setdefault(file_name, save['image'])
        del save


    

In [6]:

    

v_t = sample(range(1,9),2) + sample(range(9,17),2) + sample(range(16,25),2)
shuffle(v_t)
valid = v_t[:3]
test = v_t[3:]
train = list(set(range(1,25)) - set(v_t))

def get_names(ls):
    return ['p'+str(x) for x in ls]

train = get_names(train)
valid = get_names(valid)
test = get_names(test)

print('train',train)
print('valid',valid)
print('test',test)

def add_dic(x,y):
    return dict(x.items() + y.items())

def get_data(name_list):
    data = [dataset.get(name,False) for name in name_list]
    return reduce(add_dic,data)

# the dictionary is {name:numpy}; for example, one of the names is '30-8-1-gball-288.png'
train_dataset = get_data(train)
valid_dataset = get_data(valid)
test_dataset = get_data(test)


    

    




('train', ['p1', 'p2', 'p4', 'p6', 'p7', 'p8', 'p9', 'p10', 'p11', 'p12', 'p15', 'p16', 'p17', 'p19', 'p21', 'p22', 'p23', 'p24'])
('valid', ['p20', 'p3', 'p14'])
('test', ['p5', 'p13', 'p18'])

In [7]:

    

non_orientations = ['empty','cup','tball','pball','gball']
image_size = 40 # the image size is changed

def label_data(data):
    objects = list()
    orientations = list()
    values = list()
    for name, value in data.iteritems():
        obj = name.split('.')[0].split('-')[-2] # object name
        ori = name.split('.')[0].split('-')[-1] # orientation
        objects.append(obj)
        if obj in non_orientations:
            orientations.append(0)
        elif obj == 'gstick':
            if name.split('.')[0].split('-')[3] in ['1','3']:# this line is different
                orientations.append(0)
            else:
                orientations.append(int(ori))
        else:
            orientations.append(int(ori))
        values.append(value.reshape(image_size,image_size,1).astype(np.float32))
    return objects, orientations, values

train_objects, train_orientations, train_values = label_data(train_dataset)
valid_objects, valid_orientations, valid_values = label_data(valid_dataset)
test_objects, test_orientations, test_values = label_data(test_dataset)


    

In [8]:

    

object2value = {'empty':0,'duck':1,'cup':2,'sponge':3,'tball':4,'pball':5,'gball':6,'gstick':7,'nerf':8,'calc':9,'stapler':10}
value2object = dict((value,name) for name,value in object2value.items()) 
orientations = [18*x for x in range(20)]

def convert_objects(objects):
    obj_values = np.asarray([object2value[obj] for obj in objects])
    return (np.arange(len(object2value)) == obj_values[:,None]).astype(np.float32)

def convert_orientations(orientations):
    ori_values = np.asarray(orientations)/18%10
    return (np.arange(10) == ori_values[:,None]).astype(np.float32)

train_objects_ = convert_objects(train_objects)
valid_objects_ = convert_objects(valid_objects)
test_objects_ = convert_objects(test_objects)

train_orientations_ = convert_orientations(train_orientations)
valid_orientations_ = convert_orientations(valid_orientations)
test_orientations_ = convert_orientations(test_orientations)

train_values_ = np.asarray(train_values).astype(np.float32)
valid_values_ = np.asarray(valid_values).astype(np.float32)
test_values_ = np.asarray(test_values).astype(np.float32)


    

In [9]:

    

data_file = 'depth_data2'
save={
    'train_orientations':train_orientations_,
    'valid_orientations':valid_orientations_,
    'test_orientations':test_orientations_,
    'train_objects':train_objects_,
    'valid_objects':valid_objects_,
    'test_objects':test_objects_,
    'train_values':train_values_,
    'valid_values':valid_values_,
    'test_values':test_values_,
    'object2value':object2value,
    'value2object':value2object
}

dd.io.save('depth_data2.h5', save, compression=None)


    

In [10]:

    

def randomize(dataset, classes, angles):
    permutation = np.random.permutation(classes.shape[0])
    shuffled_dataset = dataset[permutation,:,:]
    shuffled_classes = classes[permutation]
    shuffled_angles = angles[permutation]
    return shuffled_dataset, shuffled_classes, shuffled_angles

train_dataset, train_classes, train_angles = randomize(train_values_, train_objects_, train_orientations_)
small_data = train_dataset[0:100,:,:,:]

with open('small_data','wb') as f:
    save={
        'small_data':small_data,
    }
    pickle.dump(save,f,pickle.HIGHEST_PROTOCOL)
    f.close()
    
##test
image = train_dataset[11,:,:,:].reshape(-1,image_size, image_size,1)
image = np.append(image,small_data,axis=0)
print(image.shape)


    

    




(101, 40, 40, 1)